﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Text;
using Needle.Utils.Extensions;

namespace Needle.Utils.Parser
{
    /// <summary></summary>
    public class StringTokenizer : StringTokenizer<StringToken>
    {
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="s">The string to tokenize</param>
        public StringTokenizer(string s)
            : base(s)
        { }
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="s">The string to tokenize</param>
        /// <param name="punctuators">A array of characters that can only appear in single character tokens</param>
        public StringTokenizer(string s, char[] punctuators)
            : base(s, punctuators)
        { }
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="reader">The reader to read tokens from.</param>
        public StringTokenizer(TextReader reader)
            : base(reader )
        { }
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="reader">The reader to read tokens from.</param>
        /// <param name="punctuators">A array of characters that can only appear 
        /// in single character tokens, or null to use the default punctuators.</param>
        public StringTokenizer(TextReader reader, char[] punctuators)
            : base(reader, punctuators)
        { }
        /// <summary></summary>
        protected override StringToken CreateToken(TokenType type, int line, int col, int index)
        {
            return new StringToken(Buffer.Flush(), type, line, col, index);
        }
    }

    /// <summary>
    /// Parses a stream of characters into tokens.
    /// </summary>
    [DebuggerDisplay("StringTokenizer Index:{Index} Line:{Line} Column:{Column}")]
    public abstract class StringTokenizer<T> : MarshalByRefObject, IEnumerable<T>, IDisposable
        where T : StringToken
    {

        #region static properties...

        /// <summary>
        /// Default punctuators:
        /// <para>{}[]().,:;~</para>
        /// </summary>
        public static char[] DefaultPunctuators
        {
            get { return "{}[]().,:;~".ToCharArray(); }
        }

        #endregion

        #region private fields...

        private int _col;
        private int _line;
        private int _index;

        private TextReader _reader;
        private StringBuilder _buffer;

        private char _escapeChar;
        private char[] _punctuators;

        #endregion

        #region constructors...
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="s">The string to tokenize</param>
        public StringTokenizer(string s)
            : this(new StringReader(s))
        { }
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="s">The string to tokenize</param>
        /// <param name="punctuators">A array of characters that can only appear in single character tokens</param>
        public StringTokenizer(string s, char[] punctuators)
            : this(new StringReader(s), punctuators)
        { }
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="reader">The reader to read tokens from.</param>
        public StringTokenizer(TextReader reader)
            : this(reader, null)
        { }
        /// <summary>
        /// Initializes a new StringTokenizer.
        /// </summary>
        /// <param name="reader">The reader to read tokens from.</param>
        /// <param name="punctuators">A array of characters that can only appear 
        /// in single character tokens, or null to use the default punctuators.</param>
        public StringTokenizer(TextReader reader, char[] punctuators)
        {
            Ensure.NotNull(reader, "reader");
            _col = 1;
            _line = 1;
            _index = 0;
            _escapeChar = '\\';
            _buffer = new StringBuilder();
            _reader = reader;
            _punctuators = punctuators;
        }

        #endregion

        #region public methods ...

        /// <summary>
        /// Reads the next token.
        /// </summary>
        public T NextToken()
        {
            char c = Peek();
            switch (c)
            {
                /* whitespace */
                case ' ':
                case '\u0009':
                case '\u000B':
                case '\u000C':
                    return ReadWhiteSpaceToken();
                /* new-line characters */
                case '\u000D':
                case '\u000A':
                case '\u2028':
                case '\u2029':
                    return ReadNewLineToken();
                case '\0':
                    return CreateToken(TokenType.EOF);
                default:
                    if (IsPunctuation(c))
                        return ReadPunctuationToken();
                    return ReadToken();
            }
        }

        #endregion

        #region properties...

        /// <summary>
        /// Defines the currently used escape character
        /// </summary>
        public char EscapeChar
        {
            get { return _escapeChar; }
            protected set { _escapeChar = value; }
        }
        /// <summary>
        /// The buffer into which the Read() function accumulates characters
        /// </summary>
        public StringBuilder Buffer
        {
            get { return _buffer; }
        }
        /// <summary>
        /// The TextReader from which characters are read.
        /// </summary>
        public TextReader Reader
        {
            get { return _reader; }
        }
        /// <summary>
        /// The array of characters that always produce single character tokens.
        /// </summary>
        public char[] Punctuators
        {
            get
            {
                if (_punctuators == null)
                    return new char[] { '{', '}', '[', ']', '(', ')', '.', ',', ':', ';', '~' };
                return (char[])_punctuators.Clone();
            }
            set
            {
                _punctuators = value;
            }

        }
        /// <summary>
        /// The current (1 based) column number.
        /// </summary>
        public int Column
        {
            get { return _col; }
            protected set { _col = value; }
        }
        /// <summary>
        /// The current (1 based) line number.
        /// </summary>
        public int Line
        {
            get { return _line; }
            protected set { _line = value; }
        }
        /// <summary>
        /// the 0-based index of the last character read.
        /// </summary>
        public int Index
        {
            get { return _index; }
            protected set { _index = value; }
        }

        #endregion

        #region protected methods...

        /// <summary>
        /// Reads a whitespace token, or throws if no whitespace is available.
        /// </summary>
        protected T ReadWhiteSpaceToken()
        {
            char c;
            while (true)
            {
                c = Peek();
                if (IsWhiteSpace(c))
                    Read();
                else
                {
                    if (_buffer.Length == 0)
                        throw new ParseException(Line, Column, "Expected whitespace.");
                    return CreateToken(TokenType.WhiteSpace);
                }
            }
        }
        /// <summary>
        /// Reads a newline token, or throws if no newline is available.
        /// </summary>
        protected T ReadNewLineToken()
        {
            int line = Line;
            int col = Column;
            int index = Index;
            ReadNewLine();
            return CreateToken(TokenType.NewLine, line, col, index);
        }
        /// <summary>
        /// Reads a punctuation token or throws if no more characters are available.  
        /// </summary>
        protected T ReadPunctuationToken()
        {
            if (Read() == '\0')
                throw new ParseException(Line, Column, "Expected a character.");
            return CreateToken(TokenType.Punctuator);
        }
        /// <summary>
        /// Creates a token from the current buffer contents and clears the buffer. 
        /// </summary>
        protected T CreateToken(TokenType type)
        {
            int l = _buffer.Length;
            return CreateToken(type, _line, _col - l, _index - l);
        }

        /// <summary>
        /// Parses an escape sequence starting with EscapeChar into a single 
        /// number or character. Throws if the escape sequence does not start with EscapeChar.
        /// </summary>
        protected int ParseEscapeSequence()
        {
            char c = Discard();
            if (c != _escapeChar)
                throw new ParseException(Line, Column, string.Format("Expected escape sequence starting with '{0}'", _escapeChar));

            c = Peek();
            switch (c)
            {
                /* simple escape sequences */
                case 't':
                    Discard();
                    return '\t';
                case 'n':
                    Discard();
                    return '\n';
                case 'r':
                    Discard();
                    return '\r';
                case '\\':
                    Discard();
                    return '\\';
                case 'a':
                    Discard();
                    return '\a';
                case 'b':
                    Discard();
                    return '\b';
                case 'f':
                    Discard();
                    return '\f';
                case '0':
                    Discard();
                    return '\0';
                /* hex escape sequence */
                case 'x':
                    return ParseHexEscapeSequenceInternal();
                /* Unicode escape sequence */
                case 'u':
                case 'U':
                    return ParseUnicodeEscapeSequenceInternal();
                case '\0':
                    throw new ParseException(Line, Column, "Incomplete escape sequence");
                default:
                    return ParseCustomEscapeSequence();
            }

        }
        /// <summary>
        /// Parses an unicode escape sequence of the following forms: 
        /// <para>
        /// [EscapeChar]u hex-digit hex-digit hex-digit hex-digit</para><para>
        /// [EscapeChar]U hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit
        /// </para>
        /// </summary>
        /// <returns>An integer representing the unicode character.</returns>
        protected int ParseUnicodeEscapeSequence()
        {
            char c = Discard();
            if (c != _escapeChar)
                throw new ParseException(Line, Column, string.Format("Expected escape sequence starting with '{0}'", _escapeChar));
            return ParseUnicodeEscapeSequenceInternal();
        }
        /// <summary>
        /// Parses an hexadecimal escape sequence of the following forms: 
        /// <para>
        /// [EscapeChar]x hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit
        /// </para><para>
        /// [EscapeChar]X hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit hex-digit
        /// </para>
        /// the hex digits after the first one are optional, and the maximum number of hex digits is 8.
        /// </summary>
        /// <returns>An integer representing the unicode character.</returns>
        protected int ParseHexEscapeSequence()
        {
            char c = Discard();
            if (c != _escapeChar)
                throw new ParseException(Line, Column, string.Format("Expected escape sequence starting with '{0}'", _escapeChar));
            return ParseHexEscapeSequenceInternal();
        }
        /// <summary>
        /// Returns true if the character is a hexadecimal digit (0-9, A-F or a-f)
        /// </summary>
        protected bool IsHexDigit(char c)
        {
            switch (c)
            {
                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                case '8':
                case '9':
                case 'A':
                case 'B':
                case 'C':
                case 'D':
                case 'E':
                case 'F':
                case 'a':
                case 'b':
                case 'c':
                case 'd':
                case 'e':
                case 'f':
                    return true;
                default:
                    return false;
            }
        }
        /// <summary>
        /// Returns true if the character is a punctuation character. 
        /// By default these are: 
        /// <para>{ } [ ] ( ) . , : ; ~</para>
        /// </summary>
        protected bool IsPunctuation(char c)
        {
            char[] punctuators = _punctuators;
            if (punctuators == null)
            {
                switch (c)
                {
                    case '{':
                    case '}':
                    case '[':
                    case ']':
                    case '(':
                    case ')':
                    case '.':
                    case ',':
                    case ':':
                    case ';':
                    case '~':
                        return true;
                    default:
                        return false;
                }
            }
            else
            {
                if (punctuators.Length == 0)
                {
                    return false;
                }
                else
                {
                    /* Using a simple loop here instead of a hashtable because 
                     * the number of punctuators is expected to be low 
                     * ( less than 10 ) so a hashtable would only introduce 
                     * more overhead */
                    foreach (Char p in punctuators)
                        if (c == p)
                            return true;
                }
                return false;
            }
        }
        /// <summary>
        /// Returns true if the character is a 
        /// whitespace character but not a newline character.
        /// </summary>
        protected bool IsWhiteSpace(char c)
        {
            if (IsNewLine(c))
                return false;
            return char.IsWhiteSpace(c);
        }
        /// <summary>
        /// Returns true if the character is a newline character.
        /// <para>
        /// Carriage return (U+000D or \r ), Line feed (U+000A or \n), 
        /// Line separator (U+2028) or Paragraph separator (U+2029)
        /// </para>
        /// </summary>
        protected bool IsNewLine(char c)
        {
            switch (c)
            {
                /* new-line characters */
                case '\u000D':
                case '\u000A':
                case '\u2028':
                case '\u2029':
                    return true;
                default:
                    return false;
            }
        }
        /// <summary>
        /// Reads a newline character or the \r\n combo into the buffer 
        /// and adjusts the Line and Column properties accordingly.
        /// </summary>
        protected void ReadNewLine()
        {
            char c = Read();
            switch (c)
            {
                case '\u000D':
                    if (Peek() == '\u000A')
                        Read(); /* carriage return - line feed combo */
                    break;
                case '\u000A':
                case '\u2028':
                case '\u2029':
                    break;
                default:
                    throw new ParseException(Line, Column, "Expected newline character.");
            }
            _col = 1;
            _line++;
        }
        /// <summary>
        /// Reads a character into the buffer 
        /// and adjusts the Column property accordingly.
        /// if no more characters can be read, 
        /// Read() returns the null character ( \0 ).
        /// </summary>
        protected char Read()
        {
            int x = _reader.Read();
            if (x < 0)
                return '\0';
            _col++;
            _index++;
            char c = (char)x;
            _buffer.Append(c);
            return c;
        }
        /// <summary>
        /// Returns the next character to be produced by Read().  
        /// If no more characters can be read, 
        /// Peek() returns the null character ( \0 ).
        /// </summary>
        protected char Peek()
        {
            int x = _reader.Peek();
            if (x < 0)
                return '\0';
            return (char)x;
        }
        /// <summary>
        /// Returns the last character read into the 
        /// buffer or the null character ( \0 ) if the buffer is empty.
        /// </summary>
        protected char PeekLast()
        {
            if (_buffer.Length < 1)
                return '\0';
            return _buffer[_buffer.Length - 1];
        }
        /// <summary>
        /// Reads and discards the next character. Returns the discarded 
        /// character or the null character ( \0 ) if no more characters could be read.
        /// </summary>
        protected char Discard()
        {
            int x = _reader.Read();
            if (x < 0)
                return '\0';
            _col++;
            _index++;
            return (char)x;
        }
        /// <summary>
        /// Discards the last character in the buffer. Returns the discarded 
        /// character or the null character ( \0 ) if the buffer was empty.
        /// </summary>
        protected char DiscardLast()
        {

            if (_buffer.Length < 1)
                return '\0';
            char c = _buffer[_buffer.Length - 1];
            _buffer.Length -= 1;
            return c;
        }

        #endregion

        #region extension points...

        /// <summary>
        /// Read tokens that are not a newline, whitespace or punctuator token.
        /// You can overrride this method to return more specific token types then
        /// TokenType.Token
        /// </summary>
        protected virtual T ReadToken()
        {
            char c;
            while (true)
            {
                c = Peek();
                switch (c)
                {
                    /* whitespace */
                    case ' ':
                    case '\u0009':
                    case '\u000B':
                    case '\u000C':
                    /* new-line characters */
                    case '\u000D':
                    case '\u000A':
                    case '\u2028':
                    case '\u2029':
                    /* end-of-file character */
                    case '\0':
                        return CreateToken(TokenType.Token);
                    default:
                        if (char.IsWhiteSpace(c))
                            return CreateToken(TokenType.Token);
                        if (IsPunctuation(c))
                            return CreateToken(TokenType.Token);
                        Read();
                        break;
                }
            }

        }

        /// <summary>
        /// Creates a token from the current buffer contents and clears the buffer. 
        /// Uses the given line and column to indicate  the start of the token 
        /// (you must use this overload with multi-line tokens, or the token will have a wrong
        /// line and column number)
        /// </summary>
        protected abstract T CreateToken(TokenType type, int line, int col, int index);

        /// <summary>
        /// parses a custom escape sequence into a integer.
        /// if this method is not overriden it reads and returns the next 
        /// character from the reader as is.
        /// </summary>
        protected virtual int ParseCustomEscapeSequence()
        {
            return (int)Discard();
        }

        #endregion

        #region private methods...

        private int ParseUnicodeEscapeSequenceInternal()
        {
            int l = _buffer.Length;
            char seqType = Discard();
            int maxCount;
            if (seqType == 'u')
                maxCount = 4;
            else if (seqType == 'U')
                maxCount = 8;
            else
                throw new ParseException(Line, Column, "Expected u or U for unicode escape sequence.");
            while (maxCount > 0 && IsHexDigit(Peek()))
            {
                maxCount--;
                Read();
            }
            string escSeq = _buffer.ToString(l, _buffer.Length - l);
            _buffer.Length = l;
            return int.Parse(escSeq, NumberStyles.HexNumber);
        }

        private int ParseHexEscapeSequenceInternal()
        {
            int l = _buffer.Length;
            char seqChar = Discard();
            if (seqChar != 'x' && seqChar != 'X') /* we need the x */
                throw new ParseException(Line, Column, "Expected x or X for hexadecimal escape sequence.");
            int maxCount = 8;
            while (maxCount > 0 && IsHexDigit(Peek()))
            {
                maxCount--;
                Read();
            }
            string hexSeq = _buffer.ToString(l, _buffer.Length - l);
            _buffer.Length = l;
            return int.Parse(hexSeq, NumberStyles.HexNumber);
        }

        #endregion

        #region Enumerators

        /// <summary>
        /// An enumerable that discard all the token types given from the enumeration.
        /// </summary>
        public IEnumerable<T> Excluding(params TokenType[] tokenTypes)
        {
            if (tokenTypes == null || tokenTypes.Length == 0)
                return this;
            else
                return GetExcludingEnumerator(tokenTypes);
        }

        private IEnumerable<T> GetExcludingEnumerator(params TokenType[] tokenTypes)
        {
            int l = tokenTypes.Length;
            var ok = true;
            foreach (var item in this)
            {
                TokenType t = item.Type;
                for (int i = 0; i < l; i++)
                    if (t == tokenTypes[i])
                    {
                        ok = false;
                        break;
                    }
                if (ok)
                    yield return item;
                else
                    ok = true;
            }
        }

        /// <summary>
        /// An enumerable that produces only the token types given and discards all other types.
        /// </summary>
        public IEnumerable<T> Including(params TokenType[] tokenTypes)
        {
            if (tokenTypes == null || tokenTypes.Length == 0)
                return this;
            else
                return GetIncludingEnumerator(tokenTypes);

        }

        private IEnumerable<T> GetIncludingEnumerator(params TokenType[] tokenTypes)
        {
            int l = tokenTypes.Length;
            foreach (var item in this)
            {
                TokenType t = item.Type;
                for (int i = 0; i < l; i++)
                    if (t == tokenTypes[i])
                        yield return item;
            }
        }
        /// <summary>
        /// The enumerator of all the tokens, including End-Of-File.
        /// </summary>
        public IEnumerator<T> GetEnumerator()
        {
            bool EOF = false;
            while (!EOF)
            {
                T token = NextToken();
                EOF = (TokenType.EOF == token.Type);
                yield return token;
            }
            yield break;
        }

        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
        {
            return GetEnumerator();
        }

        #endregion

        #region IDisposable Members

        /// <summary>
        /// Disposes the StringTokenizer and the internal reader.
        /// </summary>
        public void Dispose()
        {
            _reader.Dispose();
        }

        #endregion

    }

}
